{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "    http://www.apache.org/licenses/LICENSE-2.0\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1,2,3\"\n",
    "start = time.time()\n",
    "very_start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.14.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import pandas as pd, \n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "#pd.set_option('display.max_columns', 500)\n",
    "#pd.set_option('display.max_rows', 500)\n",
    "import cudf, cupy, time, rmm\n",
    "cudf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask as dask, dask_cudf\n",
    "from dask.distributed import Client, wait\n",
    "from dask_cuda import LocalCUDACluster\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169.254.8.236\n"
     ]
    }
   ],
   "source": [
    "cmd = \"hostname --all-ip-addresses\"\n",
    "process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)\n",
    "output, error = process.communicate()\n",
    "IPADDR = str(output.decode()).split()[0]\n",
    "print(IPADDR)\n",
    "# this is not the correct ip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>ucx://10.2.61.36:49857</li>\n",
       "  <li><b>Dashboard: </b><a href='http://10.2.61.36:8787/status' target='_blank'>http://10.2.61.36:8787/status</a></li>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>4</li>\n",
       "  <li><b>Memory: </b>270.39 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: 'ucx://10.2.61.36:49857' processes=4 threads=4, memory=270.39 GB>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster = LocalCUDACluster(ip='10.2.61.36',protocol=\"ucx\", \n",
    "                           rmm_pool_size=\"26GB\",\n",
    "                           enable_tcp_over_ucx=True, enable_nvlink=True)\n",
    "#cluster = LocalCUDACluster()\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#client.run(cudf.set_allocator, pool=True, initial_pool_size=int(2**35)-int(2**32))\n",
    "#client.run(cupy.cuda.set_allocator, rmm.rmm_cupy_allocator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 846 ms, sys: 302 ms, total: 1.15 s\n",
      "Wall time: 1.14 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "path = '/raid/data/recsys/train_split'\n",
    "train = dask_cudf.read_parquet(f'{path}/train-preproc-fold-*.parquet')#,dtypes=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 120 ms, sys: 19.9 ms, total: 140 ms\n",
      "Wall time: 127 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# DROP UNUSED COLUMNS\n",
    "cols_drop = ['links','hashtags0', 'hashtags1', 'fold', 'a_account_creation',\n",
    " 'b_account_creation']\n",
    "train = train.drop(cols_drop,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-b218d7b2-71f1-46d1-bdb6-8a5543cdbe4d'), 23)\n",
      "CPU times: user 9.54 ms, sys: 0 ns, total: 9.54 ms\n",
      "Wall time: 8.87 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 131 ms, sys: 29.7 ms, total: 161 ms\n",
      "Wall time: 1.22 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>92560312</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>2020-02-09 09:26:50</td>\n",
       "      <td>0</td>\n",
       "      <td>14326</td>\n",
       "      <td>408</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55533709</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>2020-02-09 18:41:35</td>\n",
       "      <td>10</td>\n",
       "      <td>237126</td>\n",
       "      <td>1193</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37019981</th>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020-02-09 01:13:28</td>\n",
       "      <td>19</td>\n",
       "      <td>23079</td>\n",
       "      <td>1803</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74047136</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>2020-02-07 12:15:20</td>\n",
       "      <td>29</td>\n",
       "      <td>769176</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-02-07 12:36:47</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74047139</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>2020-02-08 14:14:39</td>\n",
       "      <td>35</td>\n",
       "      <td>73952</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-02-09 13:33:47</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          tweet_id  media  domains  tweet_type  language           timestamp  \\\n",
       "92560312         0      7        0           1        54 2020-02-09 09:26:50   \n",
       "55533709        10      0        0           1        54 2020-02-09 18:41:35   \n",
       "37019981        20      5        0           1         3 2020-02-09 01:13:28   \n",
       "74047136        30      0        5           2        11 2020-02-07 12:15:20   \n",
       "74047139        40      0        0           2         6 2020-02-08 14:14:39   \n",
       "\n",
       "          a_user_id  a_follower_count  a_following_count  a_is_verified  ...  \\\n",
       "92560312          0             14326                408          False  ...   \n",
       "55533709         10            237126               1193           True  ...   \n",
       "37019981         19             23079               1803          False  ...   \n",
       "74047136         29            769176                190          False  ...   \n",
       "74047139         35             73952                 13          False  ...   \n",
       "\n",
       "          b_is_verified  b_follows_a  reply  retweet  retweet_comment  like  \\\n",
       "92560312          False         True      0        0                0     0   \n",
       "55533709          False        False      0        0                0     0   \n",
       "37019981          False        False      0        0                0     0   \n",
       "74047136          False        False      0        0                0     1   \n",
       "74047139          False        False      0        0                0     1   \n",
       "\n",
       "                 engage_time len_domains  len_hashtags  len_links  \n",
       "92560312 1970-01-01 00:00:00           0             0          0  \n",
       "55533709 1970-01-01 00:00:00           0             0          0  \n",
       "37019981 1970-01-01 00:00:00           0             0          0  \n",
       "74047136 2020-02-07 12:36:47           0             0          0  \n",
       "74047139 2020-02-09 13:33:47           0             0          0  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.repartition(npartitions=4)\n",
    "train, = dask.persist(train)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 29629913\n",
      "1 44309248\n",
      "2 29638018\n",
      "3 44498059\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(i, len(train.partitions[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = ['reply', 'retweet', 'retweet_comment', 'like']\n",
    "for col in train.columns:\n",
    "    if col in label_names:\n",
    "        train[col] = train[col].astype('float32')\n",
    "    elif train[col].dtype=='int64':\n",
    "        train[col] = train[col].astype('int32')\n",
    "    elif train[col].dtype=='int16':\n",
    "        train[col] = train[col].astype('int8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11 ms, sys: 0 ns, total: 11 ms\n",
      "Wall time: 9.75 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train = train.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Delayed('int-3c3a6355-9e7d-4fea-b49e-546ebee424b4'), 23)\n",
      "CPU times: user 51.4 ms, sys: 0 ns, total: 51.4 ms\n",
      "Wall time: 50.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 79.4 ms, sys: 14.9 ms, total: 94.3 ms\n",
      "Wall time: 86.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# TIME FEATURES\n",
    "# RAPIDS does this 5x faster than Pandas CPU\n",
    "# If we didn't need to copy CPU to GPU to CPU, then 1300x faster!\n",
    "def split_time(df):\n",
    "    #gf = cudf.from_pandas(df[['timestamp']])\n",
    "    df['dt_dow']  = df['timestamp'].dt.weekday#.to_array() \n",
    "    df['dt_hour'] = df['timestamp'].dt.hour#.to_array()\n",
    "    df['dt_minute'] = df['timestamp'].dt.minute#.to_array()\n",
    "    df['dt_second'] = df['timestamp'].dt.second#.to_array()\n",
    "    return df\n",
    "\n",
    "train = split_time(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>engage_time</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 09:26:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 18:41:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 01:13:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-02-07 12:36:47</td>\n",
       "      <td>2020-02-07 12:15:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-02-09 13:33:47</td>\n",
       "      <td>2020-02-08 14:14:39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          engage_time           timestamp\n",
       "0 1970-01-01 00:00:00 2020-02-09 09:26:50\n",
       "1 1970-01-01 00:00:00 2020-02-09 18:41:35\n",
       "2 1970-01-01 00:00:00 2020-02-09 01:13:28\n",
       "3 2020-02-07 12:36:47 2020-02-07 12:15:20\n",
       "4 2020-02-09 13:33:47 2020-02-08 14:14:39"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()[['engage_time','timestamp']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('<M8[ns]')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.timestamp.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.8 ms, sys: 0 ns, total: 23.8 ms\n",
      "Wall time: 22.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# ELAPSED TIME\n",
    "for col in ['engage_time','timestamp']:\n",
    "    train[col] = train[col].astype('int64')/1e3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-622cd132-88a6-4826-a17c-876bb9ca991d'), 27)\n",
      "CPU times: user 26.5 ms, sys: 3.79 ms, total: 30.3 ms\n",
      "Wall time: 29.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>engage_time</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581240e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581274e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581211e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.581079e+09</td>\n",
       "      <td>1.581078e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.581255e+09</td>\n",
       "      <td>1.581171e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    engage_time     timestamp\n",
       "0  0.000000e+00  1.581240e+09\n",
       "1  0.000000e+00  1.581274e+09\n",
       "2  0.000000e+00  1.581211e+09\n",
       "3  1.581079e+09  1.581078e+09\n",
       "4  1.581255e+09  1.581171e+09"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()[['engage_time','timestamp']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_nan(ds):\n",
    "    mask = ds == 0\n",
    "    ds.loc[mask] = np.nan\n",
    "    return ds.nans_to_nulls()\n",
    "train['engage_time'] = train['engage_time'].map_partitions(set_nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['elapsed_time'] = train['engage_time'] - train['timestamp']\n",
    "train['elapsed_time'] = train.elapsed_time.astype('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0 603956.0\n",
      "15581.699535245267\n"
     ]
    }
   ],
   "source": [
    "print(train['elapsed_time'].min().compute(),train['elapsed_time'].max().compute())\n",
    "print(train['elapsed_time'].mean().compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-65b25a28-50b1-4d9a-91f4-af47d48594a7'), 28)\n",
      "CPU times: user 12.6 ms, sys: 101 µs, total: 12.7 ms\n",
      "Wall time: 12.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1.581240e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>14326</td>\n",
       "      <td>408</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>26</td>\n",
       "      <td>50</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1.581274e+09</td>\n",
       "      <td>10</td>\n",
       "      <td>237126</td>\n",
       "      <td>1193</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>41</td>\n",
       "      <td>35</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.581211e+09</td>\n",
       "      <td>19</td>\n",
       "      <td>23079</td>\n",
       "      <td>1803</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>28</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1.581078e+09</td>\n",
       "      <td>29</td>\n",
       "      <td>769176</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581079007e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>1287.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1.581171e+09</td>\n",
       "      <td>35</td>\n",
       "      <td>73952</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581255227e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>39</td>\n",
       "      <td>83948.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0         0      7        0           1        54  1.581240e+09          0   \n",
       "1        10      0        0           1        54  1.581274e+09         10   \n",
       "2        20      5        0           1         3  1.581211e+09         19   \n",
       "3        30      0        5           2        11  1.581078e+09         29   \n",
       "4        40      0        0           2         6  1.581171e+09         35   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ...  like  \\\n",
       "0             14326                408          False  ...   0.0   \n",
       "1            237126               1193           True  ...   0.0   \n",
       "2             23079               1803          False  ...   0.0   \n",
       "3            769176                190          False  ...   1.0   \n",
       "4             73952                 13          False  ...   1.0   \n",
       "\n",
       "       engage_time len_domains  len_hashtags  len_links  dt_dow  dt_hour  \\\n",
       "0             null           0             0          0       6        9   \n",
       "1             null           0             0          0       6       18   \n",
       "2             null           0             0          0       6        1   \n",
       "3  1.581079007e+09           0             0          0       4       12   \n",
       "4  1.581255227e+09           0             0          0       5       14   \n",
       "\n",
       "   dt_minute  dt_second  elapsed_time  \n",
       "0         26         50          null  \n",
       "1         41         35          null  \n",
       "2         13         28          null  \n",
       "3         15         20        1287.0  \n",
       "4         14         39       83948.0  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47 ms, sys: 300 µs, total: 47.3 ms\n",
      "Wall time: 45.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# TRAIN FIRST 5 DAYS. VALIDATE LAST 2 DAYS\n",
    "VALID_DOW = [1, 2]# order is [3, 4, 5, 6, 0, 1, 2]\n",
    "valid = train[train['dt_dow'].isin(VALID_DOW)].reset_index(drop=True)\n",
    "train = train[~train['dt_dow'].isin(VALID_DOW)].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-83f2d59d-2f43-4a7e-8928-f40dc58f0cd9'), 28) (Delayed('int-ed8484d9-f286-4236-8e24-677b5c9baffe'), 28)\n",
      "CPU times: user 11.3 ms, sys: 0 ns, total: 11.3 ms\n",
      "Wall time: 10.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "print(type(train), train.shape, valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.19 s, sys: 3.49 s, total: 5.68 s\n",
      "Wall time: 2.09 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1173270</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>672136</td>\n",
       "      <td>5341</td>\n",
       "      <td>840</td>\n",
       "      <td>False</td>\n",
       "      <td>2407520</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>164980</td>\n",
       "      <td>5</td>\n",
       "      <td>6625</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>112668</td>\n",
       "      <td>22984</td>\n",
       "      <td>6033</td>\n",
       "      <td>True</td>\n",
       "      <td>15890909</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3240270</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>64906</td>\n",
       "      <td>256871</td>\n",
       "      <td>8</td>\n",
       "      <td>True</td>\n",
       "      <td>8602066</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2468740</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1278002</td>\n",
       "      <td>16289</td>\n",
       "      <td>376</td>\n",
       "      <td>False</td>\n",
       "      <td>14428135</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3653210</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>98180</td>\n",
       "      <td>9841</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "      <td>16369737</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language  a_user_id  \\\n",
       "0   1173270      0        4           2        54     672136   \n",
       "1    164980      5     6625           2         3     112668   \n",
       "2   3240270      9        0           2        11      64906   \n",
       "3   2468740      5        4           2        54    1278002   \n",
       "4   3653210      0        0           2        11      98180   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  b_user_id  ...  \\\n",
       "0              5341                840          False    2407520  ...   \n",
       "1             22984               6033           True   15890909  ...   \n",
       "2            256871                  8           True    8602066  ...   \n",
       "3             16289                376          False   14428135  ...   \n",
       "4              9841                  6           True   16369737  ...   \n",
       "\n",
       "   retweet_comment  like  engage_time len_domains  len_hashtags  len_links  \\\n",
       "0              0.0   0.0         null           0             0          0   \n",
       "1              0.0   0.0         null           0             0          0   \n",
       "2              0.0   0.0         null           0             2          0   \n",
       "3              0.0   0.0         null           0             0          0   \n",
       "4              0.0   0.0         null           0             5          0   \n",
       "\n",
       "   dt_hour  dt_minute  dt_second  elapsed_time  \n",
       "0        0          0          0          null  \n",
       "1        0          0          0          null  \n",
       "2        0          0          0          null  \n",
       "3        0          0          0          null  \n",
       "4        0          0          0          null  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.sort_values('timestamp').reset_index(drop=True)\n",
    "valid = valid.sort_values('timestamp').reset_index(drop=True)\n",
    "train = train.drop(['dt_dow','timestamp'],axis=1)\n",
    "valid = valid.drop(['dt_dow','timestamp'],axis=1)\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 26885321\n",
      "1 26832264\n",
      "2 26822864\n",
      "3 26696073\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(i,len(train.partitions[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 10236930\n",
      "1 10267563\n",
      "2 10191750\n",
      "3 10142473\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(i,len(valid.partitions[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MTE_one_shot:\n",
    "    \n",
    "    def __init__(self, folds, smooth, seed=42, mode='gpu'):\n",
    "        self.folds = folds\n",
    "        self.seed = seed\n",
    "        self.smooth = smooth\n",
    "        if mode=='gpu':\n",
    "            self.np = cupy\n",
    "            self.df = cudf\n",
    "        else:\n",
    "            self.np = np\n",
    "            self.df = pd\n",
    "        self.mode = mode\n",
    "        \n",
    "    def fit_transform(self, train, x_col, y_col, y_mean=None, out_col = None, out_dtype=None):\n",
    "        \n",
    "        self.y_col = y_col\n",
    "        self.np.random.seed(self.seed)\n",
    "        \n",
    "        if 'fold' not in train.columns:\n",
    "            fsize = len(train)//self.folds\n",
    "            if isinstance(train,dask_cudf.core.DataFrame):\n",
    "                #train['fold'] = train.map_partitions(lambda cudf_df: cudf_df.index%self.folds)\n",
    "                train['fold'] = 1\n",
    "                train['fold'] = train['fold'].cumsum()\n",
    "                train['fold'] = train['fold']//fsize\n",
    "                train['fold'] = train['fold']%self.folds\n",
    "            else:\n",
    "                #train['fold'] = self.np.random.randint(0,self.folds,len(train))\n",
    "                train['fold'] = (train.index.values//fsize)%self.folds\n",
    "        \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'TE_{tag}_{self.y_col}'\n",
    "        \n",
    "        if y_mean is None:\n",
    "            y_mean = train[y_col].mean()#.compute().astype('float32')\n",
    "        self.mean = y_mean\n",
    "        \n",
    "        cols = ['fold',x_col] if isinstance(x_col,str) else ['fold']+x_col\n",
    "        \n",
    "        agg_each_fold = train.groupby(cols).agg({y_col:['count','sum']}).reset_index()\n",
    "        agg_each_fold.columns = cols + ['count_y','sum_y']\n",
    "        \n",
    "        agg_all = agg_each_fold.groupby(x_col).agg({'count_y':'sum','sum_y':'sum'}).reset_index()\n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all.columns = cols + ['count_y_all','sum_y_all']\n",
    "        \n",
    "        agg_each_fold = agg_each_fold.merge(agg_all,on=x_col,how='left')\n",
    "        agg_each_fold['count_y_all'] = agg_each_fold['count_y_all'] - agg_each_fold['count_y']\n",
    "        agg_each_fold['sum_y_all'] = agg_each_fold['sum_y_all'] - agg_each_fold['sum_y']\n",
    "        agg_each_fold[out_col] = (agg_each_fold['sum_y_all']+self.smooth*self.mean)/(agg_each_fold['count_y_all']+self.smooth)\n",
    "        agg_each_fold = agg_each_fold.drop(['count_y_all','count_y','sum_y_all','sum_y'],axis=1)\n",
    "        \n",
    "        agg_all[out_col] = (agg_all['sum_y_all']+self.smooth*self.mean)/(agg_all['count_y_all']+self.smooth)\n",
    "        agg_all = agg_all.drop(['count_y_all','sum_y_all'],axis=1)\n",
    "        self.agg_all = agg_all\n",
    "        \n",
    "        train.columns\n",
    "        cols = ['fold',x_col] if isinstance(x_col,str) else ['fold']+x_col\n",
    "        train = train.merge(agg_each_fold,on=cols,how='left')\n",
    "        del agg_each_fold\n",
    "        #self.agg_each_fold = agg_each_fold\n",
    "        if self.mode=='gpu':\n",
    "            if isinstance(train,dask_cudf.core.DataFrame):\n",
    "                train[out_col] = train.map_partitions(lambda cudf_df: cudf_df[out_col].nans_to_nulls())\n",
    "            else:\n",
    "                train[out_col] = train[out_col].nans_to_nulls()\n",
    "        train[out_col] = train[out_col].fillna(self.mean)\n",
    "        \n",
    "        if out_dtype is not None:\n",
    "            train[out_col] = train[out_col].astype(out_dtype)\n",
    "        return train\n",
    "    \n",
    "    def transform(self, test, x_col, out_col = None, out_dtype=None):\n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'TE_{tag}_{self.y_col}'\n",
    "        test = test.merge(self.agg_all,on=x_col,how='left')\n",
    "        test[out_col] = test[out_col].fillna(self.mean)\n",
    "        if out_dtype is not None:\n",
    "            test[out_col] = test[out_col].astype(out_dtype)\n",
    "        return test\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.2 s, sys: 900 ms, total: 13.1 s\n",
      "Wall time: 39.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "start = time.time()\n",
    "for t in ['reply', 'retweet', 'retweet_comment', 'like']:\n",
    "    start = time.time()\n",
    "    for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "        out_col = f'TE_{c}_{t}'\n",
    "        encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "        train = encoder.fit_transform(train, c, t, out_col=out_col, out_dtype='float32')\n",
    "        valid = encoder.transform(valid, c, out_col=out_col, out_dtype='float32')\n",
    "        cols.append(out_col)\n",
    "        del encoder\n",
    "        train,valid = dask.persist(train,valid)\n",
    "        train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple Column Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2 s, sys: 38.6 ms, total: 2.04 s\n",
      "Wall time: 1.98 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols=[]\n",
    "c = ['domains','language','b_follows_a','tweet_type','media','a_is_verified']\n",
    "for t in ['reply', 'retweet', 'retweet_comment', 'like']:\n",
    "    out_col = f'TE_multi_{t}'\n",
    "    encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "    train = encoder.fit_transform(train, c, t, out_col=out_col, out_dtype='float32')\n",
    "    valid = encoder.transform(valid, c, out_col=out_col, out_dtype='float32')\n",
    "    cols.append(out_col)\n",
    "    del encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.12 s, sys: 99.4 ms, total: 1.22 s\n",
      "Wall time: 2.82 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>TE_b_user_id_retweet_comment</th>\n",
       "      <th>TE_media_like</th>\n",
       "      <th>TE_tweet_type_like</th>\n",
       "      <th>TE_language_like</th>\n",
       "      <th>TE_a_user_id_like</th>\n",
       "      <th>TE_b_user_id_like</th>\n",
       "      <th>TE_multi_reply</th>\n",
       "      <th>TE_multi_retweet</th>\n",
       "      <th>TE_multi_retweet_comment</th>\n",
       "      <th>TE_multi_like</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>323067</td>\n",
       "      <td>5</td>\n",
       "      <td>161</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34843521</td>\n",
       "      <td>245</td>\n",
       "      <td>True</td>\n",
       "      <td>23840898</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007008</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.049679</td>\n",
       "      <td>0.453557</td>\n",
       "      <td>0.002379</td>\n",
       "      <td>0.01318</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.079467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>323067</td>\n",
       "      <td>5</td>\n",
       "      <td>161</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34843521</td>\n",
       "      <td>245</td>\n",
       "      <td>True</td>\n",
       "      <td>24016535</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006703</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.049679</td>\n",
       "      <td>0.477315</td>\n",
       "      <td>0.002379</td>\n",
       "      <td>0.01318</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.079467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>323067</td>\n",
       "      <td>5</td>\n",
       "      <td>161</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34843521</td>\n",
       "      <td>245</td>\n",
       "      <td>True</td>\n",
       "      <td>23892268</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007341</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.049679</td>\n",
       "      <td>0.427535</td>\n",
       "      <td>0.002379</td>\n",
       "      <td>0.01318</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.079467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>323067</td>\n",
       "      <td>5</td>\n",
       "      <td>161</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34843521</td>\n",
       "      <td>245</td>\n",
       "      <td>True</td>\n",
       "      <td>24082276</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006424</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.049679</td>\n",
       "      <td>0.415760</td>\n",
       "      <td>0.002379</td>\n",
       "      <td>0.01318</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.079467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>323067</td>\n",
       "      <td>5</td>\n",
       "      <td>161</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34919640</td>\n",
       "      <td>243</td>\n",
       "      <td>True</td>\n",
       "      <td>23879187</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007708</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.049679</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.002379</td>\n",
       "      <td>0.01318</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.079467</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language  a_user_id  \\\n",
       "0    323067      5      161           2        54       1579   \n",
       "1    323067      5      161           2        54       1579   \n",
       "2    323067      5      161           2        54       1579   \n",
       "3    323067      5      161           2        54       1579   \n",
       "4    323067      5      161           2        54       1579   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  b_user_id  ...  \\\n",
       "0          34843521                245           True   23840898  ...   \n",
       "1          34843521                245           True   24016535  ...   \n",
       "2          34843521                245           True   23892268  ...   \n",
       "3          34843521                245           True   24082276  ...   \n",
       "4          34919640                243           True   23879187  ...   \n",
       "\n",
       "   TE_b_user_id_retweet_comment  TE_media_like  TE_tweet_type_like  \\\n",
       "0                      0.007008       0.494331            0.532514   \n",
       "1                      0.006703       0.494331            0.532514   \n",
       "2                      0.007341       0.494331            0.532514   \n",
       "3                      0.006424       0.494331            0.532514   \n",
       "4                      0.007708       0.494331            0.532514   \n",
       "\n",
       "   TE_language_like  TE_a_user_id_like  TE_b_user_id_like  TE_multi_reply  \\\n",
       "0          0.449791           0.049679           0.453557        0.002379   \n",
       "1          0.449791           0.049679           0.477315        0.002379   \n",
       "2          0.449791           0.049679           0.427535        0.002379   \n",
       "3          0.449791           0.049679           0.415760        0.002379   \n",
       "4          0.449791           0.049679           0.448912        0.002379   \n",
       "\n",
       "   TE_multi_retweet  TE_multi_retweet_comment  TE_multi_like  \n",
       "0           0.01318                  0.000506       0.079467  \n",
       "1           0.01318                  0.000506       0.079467  \n",
       "2           0.01318                  0.000506       0.079467  \n",
       "3           0.01318                  0.000506       0.079467  \n",
       "4           0.01318                  0.000506       0.079467  \n",
       "\n",
       "[5 rows x 51 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elapsed Time Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 970 ms, sys: 28.5 ms, total: 999 ms\n",
      "Wall time: 967 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language']:#, 'a_user_id', 'b_user_id']:\n",
    "    for t in ['elapsed_time']:\n",
    "        out_col = f'TE_{c}_{t}'\n",
    "        encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "        train = encoder.fit_transform(train, c, t, out_col=out_col)\n",
    "        out_dtype='float32' #if 'user_id' in c else None\n",
    "        valid = encoder.transform(valid, c, out_col=out_col, out_dtype=out_dtype)\n",
    "        cols.append(out_col)\n",
    "        #del encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 656 ms, sys: 73.7 ms, total: 729 ms\n",
      "Wall time: 1.68 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>TE_language_like</th>\n",
       "      <th>TE_a_user_id_like</th>\n",
       "      <th>TE_b_user_id_like</th>\n",
       "      <th>TE_multi_reply</th>\n",
       "      <th>TE_multi_retweet</th>\n",
       "      <th>TE_multi_retweet_comment</th>\n",
       "      <th>TE_multi_like</th>\n",
       "      <th>TE_media_elapsed_time</th>\n",
       "      <th>TE_tweet_type_elapsed_time</th>\n",
       "      <th>TE_language_elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>223305</td>\n",
       "      <td>0</td>\n",
       "      <td>411</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1452</td>\n",
       "      <td>3444937</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>27109478</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449679</td>\n",
       "      <td>0.633323</td>\n",
       "      <td>0.453557</td>\n",
       "      <td>0.009299</td>\n",
       "      <td>0.113452</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.502925</td>\n",
       "      <td>15701.587078</td>\n",
       "      <td>18450.83857</td>\n",
       "      <td>15133.940503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>223305</td>\n",
       "      <td>0</td>\n",
       "      <td>411</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1452</td>\n",
       "      <td>3454699</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>27279981</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449679</td>\n",
       "      <td>0.633323</td>\n",
       "      <td>0.519130</td>\n",
       "      <td>0.009299</td>\n",
       "      <td>0.113452</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.502925</td>\n",
       "      <td>15701.587078</td>\n",
       "      <td>18450.83857</td>\n",
       "      <td>15133.940503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>223305</td>\n",
       "      <td>0</td>\n",
       "      <td>411</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1452</td>\n",
       "      <td>3454699</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>4609600</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449679</td>\n",
       "      <td>0.633323</td>\n",
       "      <td>0.537625</td>\n",
       "      <td>0.009299</td>\n",
       "      <td>0.113452</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.502925</td>\n",
       "      <td>15701.587078</td>\n",
       "      <td>18450.83857</td>\n",
       "      <td>15133.940503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>223305</td>\n",
       "      <td>0</td>\n",
       "      <td>411</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1452</td>\n",
       "      <td>3454699</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>27215638</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449679</td>\n",
       "      <td>0.633323</td>\n",
       "      <td>0.475155</td>\n",
       "      <td>0.009299</td>\n",
       "      <td>0.113452</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.502925</td>\n",
       "      <td>15701.587078</td>\n",
       "      <td>18450.83857</td>\n",
       "      <td>15133.940503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>223305</td>\n",
       "      <td>0</td>\n",
       "      <td>411</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1452</td>\n",
       "      <td>3454699</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>27119370</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449679</td>\n",
       "      <td>0.633323</td>\n",
       "      <td>0.427535</td>\n",
       "      <td>0.009299</td>\n",
       "      <td>0.113452</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.502925</td>\n",
       "      <td>15701.587078</td>\n",
       "      <td>18450.83857</td>\n",
       "      <td>15133.940503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language  a_user_id  \\\n",
       "0    223305      0      411           2         4       1452   \n",
       "1    223305      0      411           2         4       1452   \n",
       "2    223305      0      411           2         4       1452   \n",
       "3    223305      0      411           2         4       1452   \n",
       "4    223305      0      411           2         4       1452   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  b_user_id  ...  \\\n",
       "0           3444937                 26           True   27109478  ...   \n",
       "1           3454699                 26           True   27279981  ...   \n",
       "2           3454699                 26           True    4609600  ...   \n",
       "3           3454699                 26           True   27215638  ...   \n",
       "4           3454699                 26           True   27119370  ...   \n",
       "\n",
       "   TE_language_like  TE_a_user_id_like  TE_b_user_id_like  TE_multi_reply  \\\n",
       "0          0.449679           0.633323           0.453557        0.009299   \n",
       "1          0.449679           0.633323           0.519130        0.009299   \n",
       "2          0.449679           0.633323           0.537625        0.009299   \n",
       "3          0.449679           0.633323           0.475155        0.009299   \n",
       "4          0.449679           0.633323           0.427535        0.009299   \n",
       "\n",
       "   TE_multi_retweet  TE_multi_retweet_comment  TE_multi_like  \\\n",
       "0          0.113452                  0.003633       0.502925   \n",
       "1          0.113452                  0.003633       0.502925   \n",
       "2          0.113452                  0.003633       0.502925   \n",
       "3          0.113452                  0.003633       0.502925   \n",
       "4          0.113452                  0.003633       0.502925   \n",
       "\n",
       "   TE_media_elapsed_time  TE_tweet_type_elapsed_time  TE_language_elapsed_time  \n",
       "0           15701.587078                 18450.83857              15133.940503  \n",
       "1           15701.587078                 18450.83857              15133.940503  \n",
       "2           15701.587078                 18450.83857              15133.940503  \n",
       "3           15701.587078                 18450.83857              15133.940503  \n",
       "4           15701.587078                 18450.83857              15133.940503  \n",
       "\n",
       "[5 rows x 54 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>TE_language_like</th>\n",
       "      <th>TE_a_user_id_like</th>\n",
       "      <th>TE_b_user_id_like</th>\n",
       "      <th>TE_multi_reply</th>\n",
       "      <th>TE_multi_retweet</th>\n",
       "      <th>TE_multi_retweet_comment</th>\n",
       "      <th>TE_multi_like</th>\n",
       "      <th>TE_media_elapsed_time</th>\n",
       "      <th>TE_tweet_type_elapsed_time</th>\n",
       "      <th>TE_language_elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>2824</td>\n",
       "      <td>31861000</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>23342181</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.415760</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>2824</td>\n",
       "      <td>31852634</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>22730399</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.390358</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>2824</td>\n",
       "      <td>31861000</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>4306434</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.579943</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>2824</td>\n",
       "      <td>31861000</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>23160229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.427535</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>2824</td>\n",
       "      <td>31852634</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>23401988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.399130</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language  a_user_id  \\\n",
       "0     61737      5        0           2        50       2824   \n",
       "1     61737      5        0           2        50       2824   \n",
       "2     61737      5        0           2        50       2824   \n",
       "3     61737      5        0           2        50       2824   \n",
       "4     61737      5        0           2        50       2824   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  b_user_id  ...  \\\n",
       "0          31861000                 80           True   23342181  ...   \n",
       "1          31852634                 80           True   22730399  ...   \n",
       "2          31861000                 80           True    4306434  ...   \n",
       "3          31861000                 80           True   23160229  ...   \n",
       "4          31852634                 80           True   23401988  ...   \n",
       "\n",
       "   TE_language_like  TE_a_user_id_like  TE_b_user_id_like  TE_multi_reply  \\\n",
       "0          0.423782           0.414891           0.415760        0.016941   \n",
       "1          0.423782           0.414891           0.390358        0.016941   \n",
       "2          0.423782           0.414891           0.579943        0.016941   \n",
       "3          0.423782           0.414891           0.427535        0.016941   \n",
       "4          0.423782           0.414891           0.399130        0.016941   \n",
       "\n",
       "   TE_multi_retweet  TE_multi_retweet_comment  TE_multi_like  \\\n",
       "0          0.036669                  0.004081       0.472134   \n",
       "1          0.036669                  0.004081       0.472134   \n",
       "2          0.036669                  0.004081       0.472134   \n",
       "3          0.036669                  0.004081       0.472134   \n",
       "4          0.036669                  0.004081       0.472134   \n",
       "\n",
       "   TE_media_elapsed_time  TE_tweet_type_elapsed_time  TE_language_elapsed_time  \n",
       "0           18000.140625                19014.623047              17206.958984  \n",
       "1           18000.140625                19014.623047              17206.958984  \n",
       "2           18000.140625                19014.623047              17206.958984  \n",
       "3           18000.140625                19014.623047              17206.958984  \n",
       "4           18000.140625                19014.623047              17206.958984  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FrequencyEncoder:\n",
    "    \n",
    "    def __init__(self, seed=42, mode='gpu'):\n",
    "        self.seed = seed\n",
    "        if mode=='gpu':\n",
    "            self.np = cupy\n",
    "            self.df = cudf\n",
    "        else:\n",
    "            self.np = np\n",
    "            self.df = pd\n",
    "        self.mode = mode\n",
    "        \n",
    "    def fit_transform(self, train, x_col, c_col=None, out_col = None):\n",
    "        self.np.random.seed(self.seed)\n",
    "        if c_col is None or c_col not in train.columns:\n",
    "            c_col = 'dummy'\n",
    "            train[c_col] = 1\n",
    "            drop = True\n",
    "        else:\n",
    "            drop = False\n",
    "            \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'CE_{tag}_norm'\n",
    "            \n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all = train.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        if drop:\n",
    "            train = train.drop(c_col,axis=1)\n",
    "        agg_all.columns = cols + [out_col]\n",
    "        agg_all[out_col] = agg_all[out_col].astype('int32')\n",
    "        agg_all[out_col] = agg_all[out_col]*1.0/len(train)\n",
    "        agg_all[out_col] = agg_all[out_col].astype('float32')\n",
    "    \n",
    "        train = train.merge(agg_all,on=cols,how='left')\n",
    "        del agg_all\n",
    "        #print(train.columns)\n",
    "        if self.mode=='gpu':\n",
    "            if isinstance(train,dask_cudf.core.DataFrame):\n",
    "                train[out_col] = train.map_partitions(lambda cudf_df: cudf_df[out_col].nans_to_nulls())\n",
    "            else:\n",
    "                train[out_col] = train[out_col].nans_to_nulls()\n",
    "        return train\n",
    "    \n",
    "    def transform(self, test, x_col, c_col=None, out_col = None):\n",
    "        return self.fit_transform(test, x_col, c_col, out_col)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CountEncoder:\n",
    "    \n",
    "    def __init__(self, seed=42, mode='gpu'):\n",
    "        self.seed = seed\n",
    "        if mode=='gpu':\n",
    "            self.np = cupy\n",
    "            self.df = cudf\n",
    "        else:\n",
    "            self.np = np\n",
    "            self.df = pd\n",
    "        self.mode = mode\n",
    "        \n",
    "    def fit_transform(self, train, test, x_col, out_col = None):\n",
    "        self.np.random.seed(self.seed)\n",
    "        \n",
    "        common_cols = [i for i in train.columns if i in test.columns and i!=x_col]\n",
    "\n",
    "        if len(common_cols):\n",
    "            c_col = common_cols[0]\n",
    "            drop = False\n",
    "        else:\n",
    "            c_col = 'dummy'\n",
    "            train[c_col] = 1\n",
    "            test[c_col]=1\n",
    "            drop = True\n",
    "            \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'CE_{tag}_norm'\n",
    "            \n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all = train.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        agg_all.columns = cols + [out_col]\n",
    "        \n",
    "        agg_test = test.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        agg_test.columns = cols + [out_col+'_test']\n",
    "        agg_all = agg_all.merge(agg_test,on=cols,how='left')\n",
    "        agg_all[out_col+'_test'] = agg_all[out_col+'_test'].fillna(0)\n",
    "        agg_all[out_col] = agg_all[out_col] + agg_all[out_col+'_test']\n",
    "        agg_all = agg_all.drop(out_col+'_test', axis=1)\n",
    "        del agg_test\n",
    "            \n",
    "        if drop:\n",
    "            train = train.drop(c_col,axis=1)\n",
    "            test = test.drop(c_col,axis=1)\n",
    "        train = train.merge(agg_all,on=cols,how='left')\n",
    "        test = test.merge(agg_all,on=cols,how='left')\n",
    "        del agg_all\n",
    "        return train,test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.37 s, sys: 90.4 ms, total: 1.46 s\n",
      "Wall time: 4.02 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF CE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "    encoder = CountEncoder()\n",
    "    out_col = f'CE_{c}'\n",
    "    train,valid = encoder.fit_transform(train, valid, c, out_col=out_col)\n",
    "    print\n",
    "    del encoder\n",
    "    train,valid = dask.persist(train,valid)\n",
    "    train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.09 s, sys: 129 ms, total: 2.22 s\n",
      "Wall time: 3.53 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF CE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "    encoder = FrequencyEncoder()\n",
    "    out_col = f'CE_{c}_norm'\n",
    "    train = encoder.fit_transform(train, c, c_col='tweet_id', out_col=out_col)\n",
    "    valid = encoder.transform(valid, c, c_col='tweet_id', out_col=out_col)\n",
    "    cols.append(out_col)\n",
    "    del encoder\n",
    "    train,valid = dask.persist(train,valid)\n",
    "    train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Difference Encode (Lag Features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_encode_cudf_v1(train,col,tar,sft=1):\n",
    "    train[col+'_sft'] = train[col].shift(sft)\n",
    "    train[tar+'_sft'] = train[tar].shift(sft)\n",
    "    out_col = f'DE_{col}_{tar}_{sft}'\n",
    "    train[out_col] = train[tar]-train[tar+'_sft']\n",
    "    mask = '__MASK__'\n",
    "    train[mask] = train[col] == train[col+'_sft']\n",
    "    train = train.drop([col+'_sft',tar+'_sft'],axis=1)\n",
    "    train[out_col] = train[out_col]*train[mask]\n",
    "    train = train.drop(mask,axis=1)\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DE b_user_id b_follower_count 1 0.5 seconds\n",
      "DE b_user_id b_follower_count -1 0.6 seconds\n",
      "DE b_user_id b_following_count 1 0.5 seconds\n",
      "DE b_user_id b_following_count -1 0.5 seconds\n",
      "DE b_user_id language 1 0.5 seconds\n",
      "DE b_user_id language -1 0.6 seconds\n",
      "CPU times: user 2.76 s, sys: 153 ms, total: 2.91 s\n",
      "Wall time: 3.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# cuDF DE ENCODING IS FAST!!\n",
    "idx = 0; cols = []; sc = 'timestamp'\n",
    "for c in ['b_user_id']:\n",
    "    for t in ['b_follower_count','b_following_count','language']:\n",
    "        for s in [1,-1]:\n",
    "            start = time.time()\n",
    "            train = diff_encode_cudf_v1(train, col=c, tar=t, sft=s)\n",
    "            valid = diff_encode_cudf_v1(valid, col=c, tar=t, sft=s)\n",
    "            train,valid = dask.persist(train,valid)\n",
    "            train.head()\n",
    "            end = time.time(); idx += 1\n",
    "            print('DE',c,t,s,'%.1f seconds'%(end-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diff Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_lang = train[['a_user_id', 'language', 'tweet_id']].drop_duplicates()\n",
    "valid_lang = valid[['a_user_id', 'language', 'tweet_id']].drop_duplicates()\n",
    "train_lang_count = train_lang.groupby(['a_user_id', 'language']).agg({'tweet_id':'count'}).reset_index()\n",
    "valid_lang_count = valid_lang.groupby(['a_user_id', 'language']).agg({'tweet_id':'count'}).reset_index()\n",
    "train_lang_count,valid_lang_count = dask.persist(train_lang_count,valid_lang_count)\n",
    "train_lang_count.head()\n",
    "del train_lang,valid_lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 54.1 ms, sys: 11.8 ms, total: 66 ms\n",
      "Wall time: 115 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>top_language</th>\n",
       "      <th>language_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23104</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23106</td>\n",
       "      <td>3</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23106</td>\n",
       "      <td>54</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23109</td>\n",
       "      <td>54</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23110</td>\n",
       "      <td>54</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a_user_id  top_language  language_count\n",
       "0      23104             3               7\n",
       "1      23106             3             215\n",
       "2      23106            54               3\n",
       "3      23109            54               5\n",
       "4      23110            54              29"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_lang_count = train_lang_count.merge(valid_lang_count,on=['a_user_id', 'language'],how='left')\n",
    "train_lang_count['tweet_id_y'] = train_lang_count['tweet_id_y'].fillna(0)\n",
    "train_lang_count['tweet_id_x'] = train_lang_count['tweet_id_x'] + train_lang_count['tweet_id_y']\n",
    "train_lang_count = train_lang_count.drop('tweet_id_y',axis=1)\n",
    "train_lang_count.columns = ['a_user_id', 'top_language', 'language_count']\n",
    "train_lang_count, = dask.persist(train_lang_count)\n",
    "train_lang_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 56.9 ms, sys: 0 ns, total: 56.9 ms\n",
      "Wall time: 112 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>top_language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>71476</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71478</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71463</th>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71466</th>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71479</th>\n",
       "      <td>4</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       a_user_id  top_language\n",
       "71476          0             4\n",
       "71478          1            47\n",
       "71463          2            21\n",
       "71466          3            25\n",
       "71479          4            54"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_lang_count = train_lang_count.sort_values(['a_user_id', 'language_count'])\n",
    "train_lang_count['a_user_shifted'] = train_lang_count['a_user_id'].shift(1)\n",
    "train_lang_count = train_lang_count[train_lang_count['a_user_id']!=train_lang_count['a_user_shifted']]\n",
    "train_lang_count = train_lang_count.drop(['a_user_shifted','language_count'],axis=1)\n",
    "train_lang_count.columns = ['a_user_id','top_language']\n",
    "train_lang_count, = dask.persist(train_lang_count)\n",
    "train_lang_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_language(df,df_lang_count):\n",
    "    df = df.merge(df_lang_count,how='left', left_on='b_user_id', right_on='a_user_id')\n",
    "    df['nan_language'] = df['top_language'].isnull()\n",
    "    df['same_language'] = df['language'] == df['top_language']\n",
    "    df['diff_language'] = df['language'] != df['top_language']\n",
    "    df['same_language'] = df['same_language']*(1-df['nan_language'])\n",
    "    df['diff_language'] = df['diff_language']*(1-df['nan_language'])\n",
    "    df = df.drop('top_language',axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 505 ms, sys: 17.5 ms, total: 523 ms\n",
      "Wall time: 712 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id_x</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>DE_b_user_id_b_follower_count_1</th>\n",
       "      <th>DE_b_user_id_b_follower_count_-1</th>\n",
       "      <th>DE_b_user_id_b_following_count_1</th>\n",
       "      <th>DE_b_user_id_b_following_count_-1</th>\n",
       "      <th>DE_b_user_id_language_1</th>\n",
       "      <th>DE_b_user_id_language_-1</th>\n",
       "      <th>a_user_id_y</th>\n",
       "      <th>nan_language</th>\n",
       "      <th>same_language</th>\n",
       "      <th>diff_language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43130</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>8826</td>\n",
       "      <td>3864033</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>170675</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>170675</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>43130</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>8826</td>\n",
       "      <td>3864033</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>6007085</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6007085</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>43130</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>8826</td>\n",
       "      <td>3864033</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>5058499</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5058499</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>43130</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>8826</td>\n",
       "      <td>3864033</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>6473473</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6473473</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43130</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>8826</td>\n",
       "      <td>3822152</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>16270149</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language  a_user_id_x  \\\n",
       "0     43130      7        0           2        47         8826   \n",
       "1     43130      7        0           2        47         8826   \n",
       "2     43130      7        0           2        47         8826   \n",
       "3     43130      7        0           2        47         8826   \n",
       "4     43130      7        0           2        47         8826   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  b_user_id  ...  \\\n",
       "0           3864033                100           True     170675  ...   \n",
       "1           3864033                100           True    6007085  ...   \n",
       "2           3864033                100           True    5058499  ...   \n",
       "3           3864033                100           True    6473473  ...   \n",
       "4           3822152                100           True   16270149  ...   \n",
       "\n",
       "   DE_b_user_id_b_follower_count_1  DE_b_user_id_b_follower_count_-1  \\\n",
       "0                                0                                 0   \n",
       "1                                0                                 0   \n",
       "2                                0                                 0   \n",
       "3                                0                                 0   \n",
       "4                                0                                 0   \n",
       "\n",
       "   DE_b_user_id_b_following_count_1  DE_b_user_id_b_following_count_-1  \\\n",
       "0                                 0                                  0   \n",
       "1                                 0                                  0   \n",
       "2                                 0                                  0   \n",
       "3                                 0                                  0   \n",
       "4                                 0                                  0   \n",
       "\n",
       "   DE_b_user_id_language_1  DE_b_user_id_language_-1  a_user_id_y  \\\n",
       "0                        0                         0       170675   \n",
       "1                        0                         0      6007085   \n",
       "2                        0                         0      5058499   \n",
       "3                        0                         0      6473473   \n",
       "4                        0                         0         null   \n",
       "\n",
       "  nan_language  same_language  diff_language  \n",
       "0        False              0              1  \n",
       "1        False              0              1  \n",
       "2        False              0              1  \n",
       "3        False              0              1  \n",
       "4         True              0              0  \n",
       "\n",
       "[5 rows x 74 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = diff_language(train,train_lang_count)\n",
    "valid = diff_language(valid,train_lang_count)\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rate feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 70.4 ms, sys: 201 µs, total: 70.6 ms\n",
      "Wall time: 67.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# follow rate feature\n",
    "train['a_ff_rate'] = (train['a_following_count'] / train['a_follower_count']).astype('float32')\n",
    "train['b_ff_rate'] = (train['b_follower_count']  / train['b_following_count']).astype('float32')\n",
    "valid['a_ff_rate']  = (valid['a_following_count'] / valid['a_follower_count']).astype('float32')\n",
    "valid['b_ff_rate']  = (valid['b_follower_count']  / valid['b_following_count']).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "train,valid = dask.persist(train,valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train.head()\n",
    "valid.head()\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarize Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.8 ms, sys: 0 ns, total: 3.8 ms\n",
      "Wall time: 3.74 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "label_names = ['reply', 'retweet', 'retweet_comment', 'like']\n",
    "DONT_USE = ['timestamp','a_account_creation','b_account_creation','engage_time',\n",
    "            'fold','b_user_id','a_user_id', 'dt_dow',\n",
    "            'a_account_creation', 'b_account_creation', 'elapsed_time',\n",
    "             'links','domains','hashtags0','hashtags1']\n",
    "DONT_USE += label_names\n",
    "features = [c for c in train.columns if c not in DONT_USE]\n",
    "\n",
    "RMV = [c for c in DONT_USE if c in train.columns and c not in label_names]\n",
    "RMV = list(set(RMV))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30.5 ms, sys: 161 µs, total: 30.7 ms\n",
      "Wall time: 29.4 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DoneAndNotDoneFutures(done={<Future: finished, type: cudf.DataFrame, key: ('assign-0b47b881e1e4877e24ac587cc7236c0f', 1)>, <Future: finished, type: cudf.DataFrame, key: ('assign-0b47b881e1e4877e24ac587cc7236c0f', 0)>, <Future: finished, type: cudf.DataFrame, key: ('assign-0b47b881e1e4877e24ac587cc7236c0f', 2)>, <Future: finished, type: cudf.DataFrame, key: ('assign-0b47b881e1e4877e24ac587cc7236c0f', 3)>}, not_done=set())"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train = train.drop(RMV,axis=1)\n",
    "RMV = [c for c in RMV if c in valid.columns]\n",
    "valid = valid.drop(RMV,axis=1)    \n",
    "wait(train)\n",
    "wait(valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model Validate\n",
    "We will train on random 10% of first 5 days and validation on last 2 days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107236522\n",
      "10716577\n",
      "Using 66 features: 66\n",
      "CPU times: user 344 ms, sys: 31.4 ms, total: 375 ms\n",
      "Wall time: 939 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['media', 'tweet_type', 'language', 'a_user_id_x',\n",
       "       'a_follower_count', 'a_following_count', 'a_is_verified',\n",
       "       'b_follower_count', 'b_following_count', 'b_is_verified',\n",
       "       'b_follows_a', 'len_domains', 'len_hashtags', 'len_links',\n",
       "       'dt_hour', 'dt_minute', 'dt_second', 'TE_media_reply',\n",
       "       'TE_tweet_type_reply', 'TE_language_reply', 'TE_a_user_id_reply',\n",
       "       'TE_b_user_id_reply', 'TE_media_retweet', 'TE_tweet_type_retweet',\n",
       "       'TE_language_retweet', 'TE_a_user_id_retweet',\n",
       "       'TE_b_user_id_retweet', 'TE_media_retweet_comment',\n",
       "       'TE_tweet_type_retweet_comment', 'TE_language_retweet_comment',\n",
       "       'TE_a_user_id_retweet_comment', 'TE_b_user_id_retweet_comment',\n",
       "       'TE_media_like', 'TE_tweet_type_like', 'TE_language_like',\n",
       "       'TE_a_user_id_like', 'TE_b_user_id_like', 'TE_multi_reply',\n",
       "       'TE_multi_retweet', 'TE_multi_retweet_comment', 'TE_multi_like',\n",
       "       'TE_media_elapsed_time', 'TE_tweet_type_elapsed_time',\n",
       "       'TE_language_elapsed_time', 'CE_media', 'CE_tweet_type',\n",
       "       'CE_language', 'CE_a_user_id', 'CE_b_user_id', 'CE_media_norm',\n",
       "       'CE_tweet_type_norm', 'CE_language_norm', 'CE_a_user_id_norm',\n",
       "       'CE_b_user_id_norm', 'DE_b_user_id_b_follower_count_1',\n",
       "       'DE_b_user_id_b_follower_count_-1',\n",
       "       'DE_b_user_id_b_following_count_1',\n",
       "       'DE_b_user_id_b_following_count_-1', 'DE_b_user_id_language_1',\n",
       "       'DE_b_user_id_language_-1', 'a_user_id_y', 'nan_language',\n",
       "       'same_language', 'diff_language', 'a_ff_rate', 'b_ff_rate'],\n",
       "      dtype='<U33')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "SAMPLE_RATIO = 0.1\n",
    "SEED = 1\n",
    "\n",
    "if SAMPLE_RATIO < 1.0:\n",
    "    train['sample'] = train['tweet_id'].map_partitions(lambda cudf_df: cudf_df.hash_encode(stop=10))\n",
    "    print(len(train))\n",
    "    \n",
    "    train = train[train['sample']<10*SAMPLE_RATIO]\n",
    "    train, = dask.persist(train)\n",
    "    train.head()\n",
    "    print(len(train))\n",
    "\n",
    "\n",
    "Y_train = train[label_names]\n",
    "Y_train, = dask.persist(Y_train)\n",
    "Y_train.head()    \n",
    "    \n",
    "train = train.drop(['sample','tweet_id']+label_names,axis=1)\n",
    "train, = dask.persist(train)\n",
    "train.head()\n",
    "\n",
    "\n",
    "features = [c for c in train.columns if c not in DONT_USE]\n",
    "print('Using %i features:'%(len(features)),train.shape[1])\n",
    "np.asarray(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40838716\n",
      "16288926\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id_x</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>DE_b_user_id_b_following_count_1</th>\n",
       "      <th>DE_b_user_id_b_following_count_-1</th>\n",
       "      <th>DE_b_user_id_language_1</th>\n",
       "      <th>DE_b_user_id_language_-1</th>\n",
       "      <th>a_user_id_y</th>\n",
       "      <th>nan_language</th>\n",
       "      <th>same_language</th>\n",
       "      <th>diff_language</th>\n",
       "      <th>a_ff_rate</th>\n",
       "      <th>b_ff_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6312811</td>\n",
       "      <td>649</td>\n",
       "      <td>True</td>\n",
       "      <td>572</td>\n",
       "      <td>466</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6716746</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>1.227468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>True</td>\n",
       "      <td>482</td>\n",
       "      <td>341</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11486449</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>1.413490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>True</td>\n",
       "      <td>102</td>\n",
       "      <td>546</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.186813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>True</td>\n",
       "      <td>14</td>\n",
       "      <td>195</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.071795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>True</td>\n",
       "      <td>94</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.796610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   media  tweet_type  language  a_user_id_x  a_follower_count  \\\n",
       "0      0           2        54        44114           6312811   \n",
       "2      0           2        54        44114           6311325   \n",
       "3      0           2        54        44114           6311325   \n",
       "4      0           2        54        44114           6311325   \n",
       "5      0           2        54        44114           6311325   \n",
       "\n",
       "   a_following_count  a_is_verified  b_follower_count  b_following_count  \\\n",
       "0                649           True               572                466   \n",
       "2                647           True               482                341   \n",
       "3                647           True               102                546   \n",
       "4                647           True                14                195   \n",
       "5                647           True                94                118   \n",
       "\n",
       "   b_is_verified  ...  DE_b_user_id_b_following_count_1  \\\n",
       "0          False  ...                                 0   \n",
       "2          False  ...                                 0   \n",
       "3          False  ...                                 0   \n",
       "4          False  ...                                 0   \n",
       "5          False  ...                                 0   \n",
       "\n",
       "   DE_b_user_id_b_following_count_-1  DE_b_user_id_language_1  \\\n",
       "0                                  0                        0   \n",
       "2                                  0                        0   \n",
       "3                                  0                        0   \n",
       "4                                  0                        0   \n",
       "5                                  0                        0   \n",
       "\n",
       "   DE_b_user_id_language_-1  a_user_id_y nan_language  same_language  \\\n",
       "0                         0      6716746        False              1   \n",
       "2                         0     11486449        False              1   \n",
       "3                         0         null         True              0   \n",
       "4                         0         null         True              0   \n",
       "5                         0         null         True              0   \n",
       "\n",
       "   diff_language  a_ff_rate  b_ff_rate  \n",
       "0              0   0.000103   1.227468  \n",
       "2              0   0.000103   1.413490  \n",
       "3              0   0.000103   0.186813  \n",
       "4              0   0.000103   0.071795  \n",
       "5              0   0.000103   0.796610  \n",
       "\n",
       "[5 rows x 66 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SAMPLE_RATIO = 0.35 # VAL SET NOW SIZE OF TEST SET\n",
    "SEED = 1\n",
    "if SAMPLE_RATIO < 1.0:\n",
    "    print(len(valid))\n",
    "    valid['sample'] = valid['tweet_id'].map_partitions(lambda cudf_df: cudf_df.hash_encode(stop=10))\n",
    "    \n",
    "    valid = valid[valid['sample']<10*SAMPLE_RATIO]\n",
    "    valid, = dask.persist(valid)\n",
    "    valid.head()\n",
    "    print(len(valid))\n",
    "    \n",
    "Y_valid = valid[label_names]\n",
    "Y_valid, = dask.persist(Y_valid)\n",
    "Y_valid.head()\n",
    "\n",
    "valid = valid.drop(['sample','tweet_id']+label_names,axis=1)\n",
    "valid, = dask.persist(valid)\n",
    "valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGB Version 1.1.0\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "print('XGB Version',xgb.__version__)\n",
    "\n",
    "xgb_parms = { \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.1, \n",
    "    'subsample':0.8,\n",
    "    'colsample_bytree':0.3, \n",
    "    'eval_metric':'logloss',\n",
    "    'objective':'binary:logistic',\n",
    "    'tree_method':'gpu_hist',\n",
    "    'predictor' : 'gpu_predictor'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no dup :) \n",
      "X_train.shape (Delayed('int-0859ff55-4b53-43b8-8964-b2b0c9db145a'), 66)\n",
      "X_valid.shape (Delayed('int-86e3de93-21b6-4a8c-b14c-98c419f1f5c9'), 66)\n"
     ]
    }
   ],
   "source": [
    "if train.columns.duplicated().sum()>0:\n",
    "    raise Exception(f'duplicated!: { train.columns[train.columns.duplicated()] }')\n",
    "print('no dup :) ')\n",
    "print(f'X_train.shape {train.shape}')\n",
    "print(f'X_valid.shape {valid.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 286 ms, sys: 20.5 ms, total: 306 ms\n",
      "Wall time: 360 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id_x</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>DE_b_user_id_b_following_count_1</th>\n",
       "      <th>DE_b_user_id_b_following_count_-1</th>\n",
       "      <th>DE_b_user_id_language_1</th>\n",
       "      <th>DE_b_user_id_language_-1</th>\n",
       "      <th>a_user_id_y</th>\n",
       "      <th>nan_language</th>\n",
       "      <th>same_language</th>\n",
       "      <th>diff_language</th>\n",
       "      <th>a_ff_rate</th>\n",
       "      <th>b_ff_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6312811</td>\n",
       "      <td>649</td>\n",
       "      <td>1</td>\n",
       "      <td>572</td>\n",
       "      <td>466</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6716746</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>1.227468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>1</td>\n",
       "      <td>482</td>\n",
       "      <td>341</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11486449</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>1.413490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>1</td>\n",
       "      <td>102</td>\n",
       "      <td>546</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.186813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>195</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.071795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>44114</td>\n",
       "      <td>6311325</td>\n",
       "      <td>647</td>\n",
       "      <td>1</td>\n",
       "      <td>94</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.796610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   media  tweet_type  language  a_user_id_x  a_follower_count  \\\n",
       "0      0           2        54        44114           6312811   \n",
       "2      0           2        54        44114           6311325   \n",
       "3      0           2        54        44114           6311325   \n",
       "4      0           2        54        44114           6311325   \n",
       "5      0           2        54        44114           6311325   \n",
       "\n",
       "   a_following_count  a_is_verified  b_follower_count  b_following_count  \\\n",
       "0                649              1               572                466   \n",
       "2                647              1               482                341   \n",
       "3                647              1               102                546   \n",
       "4                647              1                14                195   \n",
       "5                647              1                94                118   \n",
       "\n",
       "   b_is_verified  ...  DE_b_user_id_b_following_count_1  \\\n",
       "0              0  ...                                 0   \n",
       "2              0  ...                                 0   \n",
       "3              0  ...                                 0   \n",
       "4              0  ...                                 0   \n",
       "5              0  ...                                 0   \n",
       "\n",
       "   DE_b_user_id_b_following_count_-1  DE_b_user_id_language_1  \\\n",
       "0                                  0                        0   \n",
       "2                                  0                        0   \n",
       "3                                  0                        0   \n",
       "4                                  0                        0   \n",
       "5                                  0                        0   \n",
       "\n",
       "   DE_b_user_id_language_-1  a_user_id_y nan_language  same_language  \\\n",
       "0                         0      6716746            0              1   \n",
       "2                         0     11486449            0              1   \n",
       "3                         0         null            1              0   \n",
       "4                         0         null            1              0   \n",
       "5                         0         null            1              0   \n",
       "\n",
       "   diff_language  a_ff_rate  b_ff_rate  \n",
       "0              0   0.000103   1.227468  \n",
       "2              0   0.000103   1.413490  \n",
       "3              0   0.000103   0.186813  \n",
       "4              0   0.000103   0.071795  \n",
       "5              0   0.000103   0.796610  \n",
       "\n",
       "[5 rows x 66 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in train.columns:\n",
    "    if train[col].dtype=='bool':\n",
    "        train[col] = train[col].astype('int8')\n",
    "        valid[col] = valid[col].astype('int8')\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()\n",
    "valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################\n",
      "### reply\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 14.4 seconds\n",
      "Predicting...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jiwei/anaconda3/envs/rapids0.14/lib/python3.7/site-packages/distributed/worker.py:3390: UserWarning: Large object of size 4.56 MB detected in task graph: \n",
      "  [<function predict.<locals>.mapped_predict at 0x7f ... titions>, True]\n",
      "Consider scattering large objects ahead of time\n",
      "with client.scatter to reduce scheduler burden and \n",
      "keep data on workers\n",
      "\n",
      "    future = client.submit(func, big_data)    # bad\n",
      "\n",
      "    big_future = client.scatter(big_data)     # good\n",
      "    future = client.submit(func, big_future)  # good\n",
      "  % (format_bytes(len(b)), s)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took 0.6 seconds\n",
      "#########################\n",
      "### retweet\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 14.2 seconds\n",
      "Predicting...\n",
      "Took 0.6 seconds\n",
      "#########################\n",
      "### retweet_comment\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 13.4 seconds\n",
      "Predicting...\n",
      "Took 0.5 seconds\n",
      "#########################\n",
      "### like\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 14.4 seconds\n",
      "Predicting...\n",
      "Took 0.6 seconds\n",
      "CPU times: user 4.21 s, sys: 752 ms, total: 4.97 s\n",
      "Wall time: 58.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# TRAIN AND VALIDATE\n",
    "\n",
    "NROUND = 300\n",
    "VERBOSE_EVAL = 50\n",
    "#ESR = 50\n",
    "    \n",
    "oof = np.zeros((len(valid),len(label_names)))\n",
    "preds = []\n",
    "for i in range(4):\n",
    "\n",
    "    name = label_names[i]\n",
    "    print('#'*25);print('###',name);print('#'*25)\n",
    "       \n",
    "    start = time.time(); print('Creating DMatrix...')\n",
    "    dtrain = xgb.dask.DaskDMatrix(client,data=train,label=Y_train.iloc[:, i])\n",
    "    dvalid = xgb.dask.DaskDMatrix(client,data=valid,label=Y_valid.iloc[:, i])\n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "             \n",
    "    start = time.time(); print('Training...')\n",
    "    model = xgb.dask.train(client, xgb_parms, \n",
    "                           dtrain=dtrain,\n",
    "                           #evals=[(dtrain,'train'),(dvalid,'valid')],\n",
    "                           num_boost_round=NROUND,\n",
    "                           #early_stopping_rounds=ESR,\n",
    "                           verbose_eval=VERBOSE_EVAL) \n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "        \n",
    "    start = time.time(); print('Predicting...')\n",
    "    #Y_valid[f'pred_{name}'] = xgb.dask.predict(client,model,valid)\n",
    "    #oof[:, i] += xgb.dask.predict(client,model,dvalid).compute()\n",
    "    preds.append(xgb.dask.predict(client,model,valid))\n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "        \n",
    "    del model, dtrain, dvalid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "yvalid = Y_valid[label_names].values.compute()\n",
    "#yvalid = cupy.asnumpy(yvalid)\n",
    "\n",
    "oof = cupy.array([i.values.compute() for i in preds]).T\n",
    "#oof = cupy.asnumpy(oof)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute Validation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import auc\n",
    "\n",
    "def precision_recall_curve(y_true,y_pred):\n",
    "    y_true = y_true.astype('float32')\n",
    "    ids = cupy.argsort(-y_pred) \n",
    "    y_true = y_true[ids]\n",
    "    y_pred = y_pred[ids]\n",
    "    y_pred = cupy.flip(y_pred,axis=0)\n",
    "\n",
    "    acc_one = cupy.cumsum(y_true)\n",
    "    sum_one = cupy.sum(y_true)\n",
    "    \n",
    "    precision = cupy.flip(acc_one/cupy.cumsum(cupy.ones(len(y_true))),axis=0)\n",
    "    precision[:-1] = precision[1:]\n",
    "    precision[-1] = 1.\n",
    "\n",
    "    recall = cupy.flip(acc_one/sum_one,axis=0)\n",
    "    recall[:-1] = recall[1:]\n",
    "    recall[-1] = 0\n",
    "    n = (recall==1).sum()\n",
    "    \n",
    "    return precision[n-1:],recall[n-1:],y_pred[n:]\n",
    "\n",
    "def compute_prauc(pred, gt):\n",
    "    prec, recall, thresh = precision_recall_curve(gt, pred)\n",
    "    recall, prec = cupy.asnumpy(recall), cupy.asnumpy(prec)\n",
    "    prauc = auc(recall, prec)\n",
    "    return prauc\n",
    "\n",
    "def log_loss(y_true,y_pred,eps=1e-15, normalize=True, sample_weight=None):\n",
    "    y_true = y_true.astype('int32')\n",
    "    y_pred = cupy.clip(y_pred, eps, 1 - eps)\n",
    "    if y_pred.ndim == 1:\n",
    "        y_pred = cupy.expand_dims(y_pred, axis=1)\n",
    "    if y_pred.shape[1] == 1:\n",
    "        y_pred = cupy.hstack([1 - y_pred, y_pred])\n",
    "\n",
    "    y_pred /= cupy.sum(y_pred, axis=1, keepdims=True)\n",
    "    loss = -cupy.log(y_pred)[cupy.arange(y_pred.shape[0]), y_true]\n",
    "    return _weighted_sum(loss, sample_weight, normalize).item()\n",
    "\n",
    "def _weighted_sum(sample_score, sample_weight, normalize):\n",
    "    if normalize:\n",
    "        return cupy.average(sample_score, weights=sample_weight)\n",
    "    elif sample_weight is not None:\n",
    "        return cupy.dot(sample_score, sample_weight)\n",
    "    else:\n",
    "        return sample_score.sum()\n",
    "\n",
    "# FAST METRIC FROM GIBA\n",
    "def compute_rce_fast(pred, gt):\n",
    "    cross_entropy = log_loss(gt, pred)\n",
    "    yt = np.mean(gt)     \n",
    "    strawman_cross_entropy = -(yt*np.log(yt) + (1 - yt)*np.log(1 - yt))\n",
    "    return (1.0 - cross_entropy/strawman_cross_entropy)*100.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reply                PRAUC:0.17495 RCE:18.83670\n",
      "retweet              PRAUC:0.53806 RCE:29.56952\n",
      "retweet_comment      PRAUC:0.05584 RCE:11.69343\n",
      "like                 PRAUC:0.77888 RCE:26.49255\n",
      "CPU times: user 1.07 s, sys: 424 ms, total: 1.5 s\n",
      "Wall time: 1.43 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "txt = ''\n",
    "for i in range(4):\n",
    "    prauc = compute_prauc(oof[:,i], yvalid[:, i])#.item()\n",
    "    rce   = compute_rce_fast(oof[:,i], yvalid[:, i]).item()\n",
    "    txt_ = f\"{label_names[i]:20} PRAUC:{prauc:.5f} RCE:{rce:.5f}\"\n",
    "    print(txt_)\n",
    "    txt += txt_ + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook took 2.3 minutes\n"
     ]
    }
   ],
   "source": [
    "print('This notebook took %.1f minutes'%((time.time()-very_start)/60.))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
